package Buscador.crawler;



import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import java.util.StringTokenizer;

import net.htmlparser.jericho.Element;
import net.htmlparser.jericho.HTMLElementName;
import net.htmlparser.jericho.Source;


public class ParserJericho {
		
	Source source;
		
	boolean barra;


	
	public PagData parsea(String URL,String content)
	{
		PagData pag= new PagData(URL);
		
		if (URL.endsWith("/"))
			barra=true;
		else barra=false;
	
			source = new Source(content);
			source.fullSequentialParse();
		
			//links
			List<Element> linkElements=source.getAllElements(HTMLElementName.A);
			for (Element linkElement : linkElements) 
				{
					String href=linkElement.getAttributeValue("href");
					
					if (href != null)
					{
						if ((!href.startsWith("http:"))&&(!href.startsWith("https:")))
						{
							
							if(barra)
							{//cambiar para quitar los doble //
								if(href.startsWith("/"))
									{href=href.substring(0,1);
									href=URL+href;}
								else href=URL+href;
								
							}else{
									if(href.startsWith("/"))
										href=URL+href;
									else href=URL+"/"+href;
								}
						}
                        //if (href.endsWith("/"))
                         //   href=href.substring(0,1);
                        if (href.endsWith(".com") || href.endsWith(".es") || href.endsWith(".html") || href.endsWith("/"))
                        		if ((!href.equals(pag.getUrl()))&& (!href.equals(pag.getUrl()+"/"))&&(!href.contains("@")))
                        					pag.addLink(href);
					}
				}

			StringTokenizer prueba= new StringTokenizer(source.getTextExtractor().setIncludeAttributes(true).toString()," |,.:",true);
		   
		    while (prueba.hasMoreTokens())
		    {
		    	String aux1=prueba.nextToken();
		    	if ((!aux1.equals(" "))&&(!aux1.equals("."))&&(!aux1.equals(","))&&(!aux1.equals(":"))&&(!aux1.equals("|")))
		    	//	if (!aux1.contains("?"))
		    		pag.addPalabra(aux1);
		    }
		    
		System.out.println();
	    System.out.println("la url es :");  
		System.out.println(pag.getUrl());	
		System.out.println();
        System.out.println();

        System.out.println("hipervinculos");
		      for (int i=0;i<pag.getLinks().length;i++)
        {
        System.out.println(pag.getLinks()[i]);
        }
/*
        System.out.println("palabras");

        for (int i=0;i<pag.getPalabras().size();i++)
        {
        System.out.println(pag.getPalabras().elementAt(i));
        }*/
		
		return pag;
	}
	

}
